import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.ContentHandler;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Scanner;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.poi.hwpf.HWPFDocument;
import org.apache.poi.hwpf.extractor.WordExtractor;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.html.HtmlParser;
import org.apache.tika.sax.BodyContentHandler;
import org.jsoup.Jsoup;
import org.xml.sax.SAXException;

//public class Extractor {
//	 public static void main(String[] args) throws IOException {
//
//		 File  file = new File("C:\\test.doc");
//		  FileInputStream fis = new FileInputStream(file.getAbsolutePath());
//		     HWPFDocument document = new HWPFDocument(fis);
//		     //write...////		     FileOutputStream fout = new FileOutputStream("web/Test.doc");
////	            document.write(fout);
////	            fout.close();
////	            fis.close();
//		     WordExtractor extractor = new WordExtractor(document);
//		     String [] fileData = extractor.getParagraphText();
//		     for(int i=0;i<fileData.length;i++){
//		       if(fileData[i] != null)
//		         System.out.println(fileData[i]);
//
//	     IOUtils.closeQuietly(fis);
//	 }
//}
//public class Extractor {
//	 public static void main(String[] args) throws IOException {
//		 InputStreamReader in = new InputStreamReader(new FileInputStream("C:\\test.txt"), "UTF-8"); 
//	     System.out.println( IOUtils.toString( in ) );
//	     IOUtils.closeQuietly(in);
//	 }


public class Extractor {
	 public static void main(String[] args) throws MalformedURLException, IOException, SAXException, TikaException {
		 
		String URL1 ="	http://asesorlegalcasas.appspot.com/consulta/index.html ";
		
		String FILE1 ="C:\\Users\\User\\workspace-sts-3.4.0.RELEASE\\Textractor\\src\\test.txt";
				
	 InputStreamReader in = new InputStreamReader(new URL( URL1 ).openStream(),"UTF-8");
	 String inTostring =  IOUtils.toString( in );
	 FileUtils.writeStringToFile(new File(FILE1), inTostring,"UTF-8");
	 //What the documentation says is that it will create a new File instance. This mean it will create a new instance in memory of the File class.
	 //This object will point to a file on you file system. However, if the file exists, it will not create a new file.
	 InputStream input = new FileInputStream(FILE1);
     BodyContentHandler handler = new BodyContentHandler();
     Metadata metadata = new Metadata();
     new HtmlParser().parse(input, handler, metadata, new ParseContext());
     String plainText = handler.toString();
     String output;
//     plainText.replaceAll("(\\t\\s)", "");
     int veces = 300;
	do {
		output= plainText.replaceAll("(?m)^\\s+$", " ");
		veces--;
		
	} while (veces > 0);
		     		 
//     plainText.replaceAll("(?m)^\\s+$", "");
//    		 (?m)^\s+$, [\\\\s]+
     
//    		 ("[\\s]+", " ")
     System.out.println(output);
     IOUtils.closeQuietly(input);
	 
	 
	 
	 
	 
//			 Scanner scanner = new Scanner(IOUtils.toString( in ));
//			 int linesCount = 1;  
//			 while (scanner.hasNext()) {
//				   
//				   linesCount++;
//			    System.out.println(Jsoup.parse(this).text());
//			    
//				
//			    scanner.next();
//			   }	   
//			   System.out.println(linesCount);
//			   scanner.close();
			   
			   
	 IOUtils.closeQuietly(in);
	 }
}